
#include <boost/algorithm/string.hpp>
#include <fstream>
#include "WebKit.h"
#include "WebPage.h"
#include "WebElement.h"
#include "thread_spider.h"

using namespace SpiderWebKitClient;
using namespace std;
using namespace boost::algorithm;

#define JD_SUB_COLUMN_CSSSELECTOR_JIAYONGDIANQI "#comCategorys > div.comCategorys-wrap.ui-switchable-tab > div.mc > div.ui-switchable-panel.c-tab.part1.ui-switchable-panel-selected > div.menu > div.item.fore1 > div > a"
#define JD_COLUMN_CSSSELECTOR_INFO "#sortlist > div.mc > div > ul > li > a"
#define JD_SUB_COLUMN_CSSSELECTOR_INFO_DIANNAO "#sortlist > div.mc > div:nth-child(1) > ul > li:nth-child(3) > a"

#define JD_SUB_COLUMN_CSSSELECTOR_DIANNAO "#oaCategorys > div > div.menu > div.item.fore1 > h3 > a"

#define JD_SUB_COLUMN_CSSSELECTOR_NORMAL "#sortlist > div.mc > div.item.fore1 > div > a"

#define JD_NEXTPAGE_CSSSELECTOR "div.m>div.pagin>a.next"
#define JD_PRODUCT_HREF_CSSSELECTOR "div#plist.m>ul.list-h>li>div.lh-wrap>div.p-name>a"

#define JD_DAJIADIAN_URL "http://channel.jd.com/electronic.html"
#define JD_SHOUJI_URL "http://shouji.jd.com/"
#define JD_SHUMA_URL "http://channel.jd.com/digital.html"
#define JD_DIANNAOBANGONG_URL "http://channel.jd.com/computer.html"

#define GETPAGE_METHOD 1

static int product_count = 0;
static int page_count = 0;

void collectJD()
{
    WebKit* wk = new WebKit("test");

    const char *jdColumnUrl[4] = {JD_DAJIADIAN_URL ,
                            JD_SHOUJI_URL,
                            JD_SHUMA_URL,
                            JD_DIANNAOBANGONG_URL};

    std::list<string> urls;

    WebPage* page1 = wk->getWebPage("http://www.fewqjljfewq.com");
    WebPage* page2 = wk->getWebPage("http://www.baidu.com");
    wk->destoryWebPage(page1);
    wk->destoryWebPage(page2);
    WebPage* page3 = wk->getWebPage("http://www.baidu.com");
    wk->destoryWebPage(page3);
    delete wk;

    /*
    for (int i = 0 ; i < 4; i ++) {
        int count = 0;
        WebPage *page_sub = NULL;
        WebPage *page_info = NULL;

        string url(jdColumnUrl[i]);
        WebPage *page = wk->getWebPage(url);
        usleep(200000);

        std::list<WebElement> collection;
        collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_NORMAL);

        WebElement element;
        if(!collection.size()) {
            collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_JIAYONGDIANQI);

            if (collection.empty()) {
                collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_DIANNAO);
                if (collection.empty()) {
                    wk->destoryWebPage(page);
                    continue;
                }
                element = collection.front();
                string url_sub = element.getAttribute("href");
                page_sub = wk->getWebPage(url_sub);
                collection = page_sub->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
                if (collection.empty()) {
                    wk->destoryWebPage(page);
                    continue;
                }
            }
        }

        usleep(200000);

        element = collection.front();

        url = element.getAttribute("href");

#if GETPAGE_METHOD
        page_info = wk->getWebPage(url);
#else
        // element click
        page->click(element, 50000);
        // webkit click
#endif

        count = 0;
        int times = 0;
        while(!count) {
            usleep(200000);

#if !GETPAGE_METHOD
            if (page_sub)
                collection = page_sub->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
            else if (page_info)
#endif
                collection = page_info->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
#if !GETPAGE_METHOD
            else
                collection = page->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
#endif
            count = collection.size();
            if (times > 10)
                break;
            times ++;
        }

        cout << "Sub column count:" << collection.size() << endl;
        std::list<WebElement>::iterator it;
        for (it = collection.begin(); it != collection.end(); it ++) {
            //cout << (*it).getOuterXml() << endl;
            std::string url = (*it).getAttribute("href");
            if (url != "")
                urls.push_back(url);
        }

        wk->destoryWebPage(page);
        if (page_info)
            wk->destoryWebPage(page_info);
        if (page_sub)
            wk->destoryWebPage(page_sub);
    }
*/

   // cout << "All column count:" << urls.size() << endl;
}

std::list<std::string> getJDPageInfo()
{
    WebKit* wk = new WebKit("test");

    const char *jdColumnUrl[4] = {JD_SHOUJI_URL ,
                            JD_DAJIADIAN_URL,
                            JD_SHUMA_URL,
                            JD_DIANNAOBANGONG_URL};

    std::list<string> urls;

    for (int i = 0 ; i < 4; i ++) {
        int count = 0;
        WebPage *page_sub = NULL;
        WebPage *page_info = NULL;

        string url(jdColumnUrl[i]);
        WebPage *page = wk->getWebPage(url, 100, (3*60*1000));
        if (page == NULL)
            break;
        usleep(200000);

        std::list<WebElement> collection;
        collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_NORMAL);

        WebElement element;
        if(!collection.size()) {
            collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_JIAYONGDIANQI);

            if (collection.empty()) {
                collection = page->getElementsByCssSelector(JD_SUB_COLUMN_CSSSELECTOR_DIANNAO);
                if (collection.empty()) {
                    wk->destoryWebPage(page);
                    continue;
                }
                element = collection.front();
                string url_sub = element.getAttribute("href");
                page_sub = wk->getWebPage(url_sub, 100, (3*60*1000));
                if (page_sub == NULL)
                    break;
                collection = page_sub->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
                if (collection.empty()) {
                    wk->destoryWebPage(page);
                    continue;
                }
            }
        }

        usleep(200000);

        element = collection.front();

        url = element.getAttribute("href");

#if GETPAGE_METHOD
        page_info = wk->getWebPage(url, 100, (3*60*1000));
        if (page_info == NULL)
            break;
#else
        // element click
        page->click(element, 50000);
        // webkit click
        //page_info = wk->click(element);
#endif

        count = 0;
        int times = 0;
        while(!count) {
            usleep(200000);

#if !GETPAGE_METHOD
            if (page_sub)
                collection = page_sub->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
            else if (page_info)
#endif
                collection = page_info->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
#if !GETPAGE_METHOD
            else
                collection = page->getElementsByCssSelector(JD_COLUMN_CSSSELECTOR_INFO);
#endif
            count = collection.size();
            if (times > 10)
                break;
            times ++;
        }

        cout << "Sub column count:" << collection.size() << endl;
        std::list<WebElement>::iterator it;
        for (it = collection.begin(); it != collection.end(); it ++) {
            //cout << (*it).getOuterXml() << endl;
            std::string url = (*it).getAttribute("href");
            if (url != "")
                urls.push_back(url);
        }

        if (page)
            wk->destoryWebPage(page);
        if (page_info)
            wk->destoryWebPage(page_info);
        if (page_sub)
            wk->destoryWebPage(page_sub);
    }

    delete wk;
    cout << "All column count:" << urls.size() << endl;
    return urls;
}

class JDParser
{
    public:
    JDParser()
    {
        std::list<std::string> urls = getJDPageInfo();

        std::list<std::string>::iterator it;
        for (it = urls.begin(); it != urls.end(); it ++) {
            WKRequest* req = m_spider.createRequest(*it, this, &JDParser::parse_product_page, 102);
            //req->start();
        }
        m_spider.start();
    }

    void start()
    {
        while(1) {
            usleep(500000);
            if(m_spider.requestCount() == 0)
                _exit(0);
        }
    }

    void parse_product_page(WebPage *page)
    {
        bool hasNextPage = false;
        std::list<WebElement> nextCollections = page->getElementsByCssSelector(JD_NEXTPAGE_CSSSELECTOR);

        if (nextCollections.size() <= 0) {
            int times = 0;
            while(nextCollections.size() <= 0) {
                sleep(2);
                nextCollections = page->getElementsByCssSelector(JD_NEXTPAGE_CSSSELECTOR);
                if (times > 20) {
                    std::cout << "current list page: " << page->getUrl() << std::endl;
                    printf("#####################################\n");
                    printf("### Warrning next page not exist. ###\n");
                    printf("#####################################\n");
                    break;
                }
                times ++;
            }
        }
        if (nextCollections.size() > 0) {
            WebElement element = nextCollections.front();
            std::string url = element.getAttribute("href");
            if (url == "") {
                int times = 0;
                while(url == "") {
                    usleep(100000);
                    url = element.getAttribute("href");
                    if (times > 20) {
                        std::cout << "current list page: " << page->getUrl() << std::endl;
                        printf("#####################################\n");
                        printf("### Warrning next page href is null. ###\n");
                        printf("#####################################\n");
                        break;
                    }
                    times++;
                }
            }
            if (url != "") {
                page_count ++;
                if (!boost::starts_with(url, "http://")) {
                    std::string indexUrl = page->getUrl();
                    std::string subUrl = indexUrl.substr(7);
                    std::string domain = "http://" + subUrl.substr(0, subUrl.find_first_of("/")) + "/";
                    if (indexUrl.find("?") != -1)
                        url = domain + "list.html" + url;
                    else
                        url = domain + url;
                }
                std::cout << page_count << " | next page:" << url << std::endl;
                hasNextPage = true;
                WKRequest* req = m_spider.createRequest(url, this, &JDParser::parse_product_page, 101);
            }
        }

        std::list<WebElement> collection = page->getElementsByCssSelector(JD_PRODUCT_HREF_CSSSELECTOR);
        if (hasNextPage) {
            int times = 0;
            while (collection.size() < 36) {
                sleep(1);
                collection = page->getElementsByCssSelector(JD_PRODUCT_HREF_CSSSELECTOR);
                if(times > 40) {
                    std::cout << "current list page: " << page->getUrl() << std::endl;
                    printf("########################################\n");
                    printf("### Warrning product not completed,  ###\n");
                    printf("########################################\n");
                    printf("only %ld\n", collection.size());
                    break;
                }
                times ++;
            }
        }

        std::ofstream file("product.txt", ios::app);
        if (!file.good())
            std::cout << "open file failed." << std::endl;
        if (collection.size() > 0) {
            std::list<WebElement>::iterator element_it = collection.begin();
            for (; element_it != collection.end(); ++element_it) {
                std::string url = (*element_it).getAttribute("href");
                if (url == "")
                    continue;
                boost::mutex::scoped_lock lock(m_countMutex);
                product_count ++;
                //std::cout << "[" << product_count << "]" << std::endl;
                file << url << std::endl;
                file.flush();
                //std::cout << "Write product : " << url << std::endl;
                lock.unlock();
                //WKRequest* req = m_spider.createRequest(url, this, &JDParser::parse_product, 100);
            }
        }
        file.close();
    }

    void parse_product(WebPage *page)
    {
        boost::mutex::scoped_lock lock(m_countMutex);
        product_count ++;
        printf("[ %d ] parse product\n",product_count);
        lock.unlock();
        std::string url = page->getUrl();
        std::string title = page->getTitle();
        std::cout << title << std::endl;
    }

    private:
    ThreadSpider m_spider;
    boost::mutex m_countMutex;
};

void collectJDThread()
{
    JDParser p;
    p.start();
}

